{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f57d3a70",
   "metadata": {},
   "source": [
    "给模型参数增加随机性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAURElEQVR4nO3df2yT950H8LedxE5I8jgEFrsZseBWriziR7cAwe2k7YZH1kXTWPPHNiGaIUSvzEGlmZAWqaUHV10Q+4O1Gw0nVYP+0zGlOrZrBK1yhgbt6pKSLrsQIGK69pJSbPNjfhwSYjv25/5geVpDoHF++Gvj90t6JPx8P7Y/z0P81vPTNomIgIhIAbPqBogodzGAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGWUBdPDgQSxevBiFhYWora1Fd3e3qlaISBElAfT73/8ezc3NePHFF/Hhhx9i1apVqKurQzAYVNEOESliUnEzam1tLdasWYPf/OY3AIBEIoGqqirs2LEDv/jFL9LdDhEpkp/uN4xGo+jp6UFLS4sxz2w2w+12w+fzTfqcSCSCSCRiPE4kErhx4wYWLFgAk8k05z0TUWpEBMPDw6isrITZfO8drbQH0LVr1xCPx2G325Pm2+12XLx4cdLntLa2Ys+ePeloj4hm0dDQEBYtWnTP8bQH0HS0tLSgubnZeKzrOpxOJ4aGhqBpmsLOiGgy4XAYVVVVKC0tvW9d2gNo4cKFyMvLQyAQSJofCATgcDgmfY7VaoXVar1rvqZpDCCiDPZFh0jSfhbMYrGgpqYGXq/XmJdIJOD1euFyudLdDhEppGQXrLm5GY2NjVi9ejXWrl2LX/3qVxgZGcGWLVtUtENEiigJoB/96Ee4evUqdu/eDb/fj0cffRRvv/32XQemiejBpuQ6oJkKh8Ow2WzQdZ3HgIgy0FQ/o7wXjIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMqkHECnT5/G97//fVRWVsJkMuEPf/hD0riIYPfu3XjooYdQVFQEt9uNS5cuJdXcuHEDmzZtgqZpKCsrw9atW3Hz5s0ZLQgRZZ+UA2hkZASrVq3CwYMHJx3fv38/XnnlFRw6dAhnzpxBcXEx6urqMDY2ZtRs2rQJ/f396OzsREdHB06fPo2nn356+ktBRNlJZgCAHDt2zHicSCTE4XDIL3/5S2NeKBQSq9Uqv/vd70RE5Pz58wJAPvjgA6PmxIkTYjKZ5PLly1N6X13XBYDouj6T9olojkz1Mzqrx4A++ugj+P1+uN1uY57NZkNtbS18Ph8AwOfzoaysDKtXrzZq3G43zGYzzpw5M+nrRiIRhMPhpImIst+sBpDf7wcA2O32pPl2u90Y8/v9qKioSBrPz89HeXm5UXOn1tZW2Gw2Y6qqqprNtolIkaw4C9bS0gJd141paGhIdUtENAtmNYAcDgcAIBAIJM0PBALGmMPhQDAYTBofHx/HjRs3jJo7Wa1WaJqWNBFR9pvVAFqyZAkcDge8Xq8xLxwO48yZM3C5XAAAl8uFUCiEnp4eo+bkyZNIJBKora2dzXaIKMPlp/qEmzdv4q9//avx+KOPPkJvby/Ky8vhdDqxc+dOvPTSS1i6dCmWLFmCF154AZWVldi4cSMA4Ktf/Sq++93vYtu2bTh06BBisRiamprw4x//GJWVlbO2YESUBVI9vXbq1CkBcNfU2NgoIrdPxb/wwgtit9vFarXK+vXrZWBgIOk1rl+/Lj/5yU+kpKRENE2TLVu2yPDw8Kyf4iMiNab6GTWJiCjMv2kJh8Ow2WzQdZ3Hg4gy0FQ/o1lxFoyIHkwMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEiZlH+Wh2imRq7+H8ZCn/0Md0FxGUof+keYTCaFXZEKDCBKu+uXziDQ91/GY21RNUofWgqAAZRruAtGykkiDklk3a9D0SxgAJFykogDklDdBinAACLlJDEOYQDlJAYQKSdxbgHlKgYQpZ05vyDpcXw8ikQirqgbUokBRGlXNL8S+Nwp99jI3xCPjCrsiFRJKYBaW1uxZs0alJaWoqKiAhs3bsTAwEBSzdjYGDweDxYsWICSkhI0NDQgEAgk1QwODqK+vh7z5s1DRUUFdu3ahfHx8ZkvDWUFU17BFxdRTkgpgLq6uuDxePD++++js7MTsVgMGzZswMjIiFHz3HPP4a233kJ7ezu6urrw6aef4sknnzTG4/E46uvrEY1G8d577+H111/HkSNHsHv37tlbKspot3fBeM0PAZAZCAaDAkC6urpERCQUCklBQYG0t7cbNRcuXBAA4vP5RETk+PHjYjabxe/3GzVtbW2iaZpEIpEpva+u6wJAdF2fSfukSOiT89L97/8s3Ye2SfehbXL2NY/cCgVUt0WzaKqf0RkdA9J1HQBQXl4OAOjp6UEsFoPb7TZqli1bBqfTCZ/PBwDw+XxYsWIF7Ha7UVNXV4dwOIz+/v5J3ycSiSAcDidNlL3M3AWjv5t2ACUSCezcuROPP/44li9fDgDw+/2wWCwoKytLqrXb7fD7/UbN58NnYnxibDKtra2w2WzGVFVVNd22KQMwgGjCtAPI4/Hg3LlzOHr06Gz2M6mWlhboum5MQ0NDc/6eNHdMpuQ/OxG5fTU05Zxp3Yza1NSEjo4OnD59GosWLTLmOxwORKNRhEKhpK2gQCAAh8Nh1HR3dye93sRZsomaO1mtVlit1um0SllCEjwLmotS2gISETQ1NeHYsWM4efIklixZkjReU1ODgoICeL1eY97AwAAGBwfhcrkAAC6XC319fQgGg0ZNZ2cnNE1DdXX1TJaFspYgEWcA5aKUtoA8Hg/eeOMN/PGPf0RpaalxzMZms6GoqAg2mw1bt25Fc3MzysvLoWkaduzYAZfLhXXr1gEANmzYgOrqamzevBn79++H3+/H888/D4/Hw62cXCWAjMdUd0EKpBRAbW1tAIBvfetbSfMPHz6Mn/70pwCAAwcOwGw2o6GhAZFIBHV1dXj11VeN2ry8PHR0dGD79u1wuVwoLi5GY2Mj9u7dO7MloSzGLaBcZRKRrPsilnA4DJvNBl3XoWma6nYoRaM3LuP8f/wbJP73rR6TGV9xb0P5P9SobYxmzVQ/o7wXjNKuoLAEBfNsn82QBMb+NvklGPRgYwBR+pnzYDLn3TEz6zbEaRYwgCjtTCYzTGb+6REDiBQwmc0wme7cAqJcxACitDOZJtsFo1zEAKK0M5nv3gUT3L7QlXILA4jSz2S6634wfid0bmIAUUbghYi5iQFEGeH2RYncBcs1DCDKCNwCyk0MIMoIEo9xAygHMYAoIyS4C5aTGECkgAmFZclfyzsWCkASPBOWaxhApER+YWnS40QsAm4B5R4GECnBHyckgAFEipjzp/V15PSAYQCREvxpHgIYQKQId8EIYACRIuY77oYXCG9GzUEMIEo7k8l090z+OGFOYgBRRpBEggGUgxhAlBFEGEC5iAFEmYEBlJMYQJQRuAuWm3g1GM0JEcHIyAjGxyf/mo0oLIA5H0jcHo9HR/G34GUUiWXSepPJhNLSUpj5axoPFAYQzQkRwVNPPYXu7u5Jx7/ykIZ/3bwOhZbbf4ISH8e2rVvQ+7/XJq232Wzwer1wOBxz1jOlHwOI5szVq1dx+fLlSceKMIrhWDEGoo8hlrBiSdH/4Nq1E/esHx0dRTzOXbQHTUrbs21tbVi5ciU0TYOmaXC5XDhx4oQxPjY2Bo/HgwULFqCkpAQNDQ0IBAJJrzE4OIj6+nrMmzcPFRUV2LVr1z030+nBdSuWj97wP+FK5GFciznRO7we4fEFqtuiNEspgBYtWoR9+/ahp6cHZ8+exbe//W384Ac/QH9/PwDgueeew1tvvYX29nZ0dXXh008/xZNPPmk8Px6Po76+HtFoFO+99x5ef/11HDlyBLt3757dpaKMNxYDbo5rAG5flBhNFCIqRWqbovSTGZo/f7689tprEgqFpKCgQNrb242xCxcuCADx+XwiInL8+HExm83i9/uNmra2NtE0TSKRyJTfU9d1ASC6rs+0fZoj8XhcvvGNbwj+/pNfd07ltlI58C+HZO9LZ2TPS93y8r7/lEcfWXrP+vnz58snn3yierFoiqb6GZ32MaB4PI729naMjIzA5XKhp6cHsVgMbrfbqFm2bBmcTid8Ph/WrVsHn8+HFStWwG7/7Nvw6urqsH37dvT39+NrX/taSj1cvHgRJSUl010EmkMigtHR0XuOj966hf/uaoMuX8G4WGC3fIzgNf896+PxOC5dugRd1+eiXZplN2/enFJdygHU19cHl8uFsbExlJSU4NixY6iurkZvby8sFgvKysqS6u12O/z+239Yfr8/KXwmxifG7iUSiSASiRiPw+EwAEDXdR4/ylAict+DxmPRcbx56i8A/jLl1wuHw7BYJj9NT5llZGRkSnUpB9AjjzyC3t5e6LqON998E42Njejq6kq5wVS0trZiz549d82vra2Fpmlz+t40PYlEAqWlpV9cOEX5+fmoqanBl7/85Vl7TZo7ExsJXyTlq7osFgsefvhh1NTUoLW1FatWrcLLL78Mh8OBaDSKUCiUVB8IBIxrNxwOx11nxSYe3+/6jpaWFui6bkxDQ0Optk1EGWjGl5UmEglEIhHU1NSgoKAAXq/XGBsYGMDg4CBcLhcAwOVyoa+vD8Fg0Kjp7OyEpmmorq6+53tYrVbj1P/ERETZL6VdsJaWFjzxxBNwOp0YHh7GG2+8gXfffRfvvPMObDYbtm7diubmZpSXl0PTNOzYsQMulwvr1q0DAGzYsAHV1dXYvHkz9u/fD7/fj+effx4ejwdWq3VOFpCIMldKARQMBvHUU0/hypUrsNlsWLlyJd555x185zvfAQAcOHAAZrMZDQ0NiEQiqKurw6uvvmo8Py8vDx0dHdi+fTtcLheKi4vR2NiIvXv3zu5SUUYoLi6eta1V3gf2YDKJZN/3YIbDYdhsNui6zt2xDCUiuHr1atLZy5kwm81wOBzIy8v74mJSbqqfUd4LRnPCZDKhoqJCdRuU4bhNS0TKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJTJV93AdIgIACAcDivuhIgmM/HZnPis3ktWBtD169cBAFVVVYo7IaL7GR4ehs1mu+d4VgZQeXk5AGBwcPC+C0fJwuEwqqqqMDQ0BE3TVLeTFbjOpkdEMDw8jMrKyvvWZWUAmc23D13ZbDb+UUyDpmlcbyniOkvdVDYOeBCaiJRhABGRMlkZQFarFS+++CKsVqvqVrIK11vquM7mlkm+6DwZEdEcycotICJ6MDCAiEgZBhARKcMAIiJlsjKADh48iMWLF6OwsBC1tbXo7u5W3ZIyra2tWLNmDUpLS1FRUYGNGzdiYGAgqWZsbAwejwcLFixASUkJGhoaEAgEkmoGBwdRX1+PefPmoaKiArt27cL4+Hg6F0WZffv2wWQyYefOncY8rrM0kSxz9OhRsVgs8tvf/lb6+/tl27ZtUlZWJoFAQHVrStTV1cnhw4fl3Llz0tvbK9/73vfE6XTKzZs3jZpnnnlGqqqqxOv1ytmzZ2XdunXy2GOPGePj4+OyfPlycbvd8uc//1mOHz8uCxculJaWFhWLlFbd3d2yePFiWblypTz77LPGfK6z9Mi6AFq7dq14PB7jcTwel8rKSmltbVXYVeYIBoMCQLq6ukREJBQKSUFBgbS3txs1Fy5cEADi8/lEROT48eNiNpvF7/cbNW1tbaJpmkQikfQuQBoNDw/L0qVLpbOzU775zW8aAcR1lj5ZtQsWjUbR09MDt9ttzDObzXC73fD5fAo7yxy6rgP47Ibdnp4exGKxpHW2bNkyOJ1OY535fD6sWLECdrvdqKmrq0M4HEZ/f38au08vj8eD+vr6pHUDcJ2lU1bdjHrt2jXE4/Gk/3QAsNvtuHjxoqKuMkcikcDOnTvx+OOPY/ny5QAAv98Pi8WCsrKypFq73Q6/32/UTLZOJ8YeREePHsWHH36IDz744K4xrrP0yaoAovvzeDw4d+4c/vSnP6luJaMNDQ3h2WefRWdnJwoLC1W3k9Oyahds4cKFyMvLu+tsRCAQgMPhUNRVZmhqakJHRwdOnTqFRYsWGfMdDgei0ShCoVBS/efXmcPhmHSdTow9aHp6ehAMBvH1r38d+fn5yM/PR1dXF1555RXk5+fDbrdznaVJVgWQxWJBTU0NvF6vMS+RSMDr9cLlcinsTB0RQVNTE44dO4aTJ09iyZIlSeM1NTUoKChIWmcDAwMYHBw01pnL5UJfXx+CwaBR09nZCU3TUF1dnZ4FSaP169ejr68Pvb29xrR69Wps2rTJ+DfXWZqoPgqeqqNHj4rVapUjR47I+fPn5emnn5aysrKksxG5ZPv27WKz2eTdd9+VK1euGNPo6KhR88wzz4jT6ZSTJ0/K2bNnxeVyicvlMsYnTilv2LBBent75e2335YvfelLOXVK+fNnwUS4ztIl6wJIROTXv/61OJ1OsVgssnbtWnn//fdVt6QMgEmnw4cPGzW3bt2Sn/3sZzJ//nyZN2+e/PCHP5QrV64kvc7HH38sTzzxhBQVFcnChQvl5z//ucRisTQvjTp3BhDXWXrw6ziISJmsOgZERA8WBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKTM/wMTSd2P1CPx8gAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "98108dee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 2])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class Model(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.fc = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 64),\n",
    "            torch.nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        #输出层参数的均值和标准差\n",
    "        self.weight_mean = torch.nn.Parameter(torch.randn(64, 2))\n",
    "        self.weight_std = torch.nn.Parameter(torch.randn(64, 2))\n",
    "\n",
    "        self.bias_mean = torch.nn.Parameter(torch.randn(2))\n",
    "        self.bias_std = torch.nn.Parameter(torch.randn(2))\n",
    "        \n",
    "    def forward(self, state):\n",
    "        state = self.fc(state)\n",
    "\n",
    "        #正态分布投影,获取输出层的参数\n",
    "        weight = self.weight_mean + torch.randn(64, 2) * self.weight_std\n",
    "        bias = self.bias_mean + torch.randn(2) * self.bias_std\n",
    "\n",
    "        #运行模式下不需要随机性\n",
    "        if not self.training:\n",
    "            weight = self.weight_mean\n",
    "            bias = self.bias_mean\n",
    "\n",
    "        #计算输出\n",
    "        return state.matmul(weight) + bias\n",
    "\n",
    "\n",
    "model = Model()\n",
    "model_delay = Model()\n",
    "\n",
    "#复制参数\n",
    "model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "model(torch.randn(5, 4)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\appDir\\python3.10\\lib\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  if not isinstance(terminated, (bool, np.bool8)):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-988.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        #因为模型本身有随机性,所以这里不需要再随机动作\n",
    "        action = model(torch.FloatTensor(state).reshape(1, 4)).argmax().item()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bfcfd21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ipykernel_13404\\2282743005.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ..\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(207,\n",
       " (array([-0.02445634,  0.00993387, -0.00453326, -0.04899614], dtype=float32),\n",
       "  0,\n",
       "  1.0,\n",
       "  array([-0.02425767, -0.18512279, -0.00551318,  0.24225307], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "018c4b1e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 417 -991.55\n",
      "weight= -0.0026646796613931656 1.476675033569336 bias= [0.19028139114379883, 1.218591570854187]\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    model_delay.eval()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(100):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #计算value\n",
    "            value = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #计算target\n",
    "            with torch.no_grad():\n",
    "                target = model_delay(next_state)\n",
    "            target = target.max(dim=1)[0].reshape(-1, 1)\n",
    "            target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "            loss = loss_fn(value, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        #复制参数\n",
    "        if (epoch + 1) % 5 == 0:\n",
    "            model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            model.eval()\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            model.train()\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "            weight = model.weight_mean + 1 * model.weight_std\n",
    "            bias = model.bias_mean + 1 * model.bias_std\n",
    "            print('weight=',\n",
    "                  weight.mean().item(),\n",
    "                  weight.std().item(), 'bias=', bias.tolist())\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVuUlEQVR4nO3dfWxb9bkH8O+xYzuvx2lSYjc0vu0GW8ntCyNtUw+JTZA1sGxaR/7YUAcV6oVL51SUcCstEhSVy1VQ0b1sbF34Y4KiK7GioHWIqICyFFJtuG+B6Kahzcq9hWRtbfeFHDtvdmI/94+S05qmJW5T/+r4+5GOxPn9HsfPOeAv9jk+x5qICIiIFLCoboCIshcDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlFEWQNu3b8eCBQuQm5uL6upqHDhwQFUrRKSIkgB644030NjYiGeeeQYfffQRli1bhtraWoRCIRXtEJEimoqLUaurq7FixQr87ne/AwAkEglUVFRg48aN+NWvfpXudohIkZx0P2EsFkNXVxeamprMMYvFgpqaGvj9/ikfE41GEY1GzfVEIoFz586htLQUmqZd956JKDUigkgkgvLyclgsl/+glfYAOnPmDOLxOFwuV9K4y+XC0aNHp3xMc3Mztm7dmo72iGgGDQwMYP78+ZedT3sAXY2mpiY0Njaa64ZhwOPxYGBgALquK+yMiKYSDodRUVGBoqKiK9alPYDmzp0Lq9WKYDCYNB4MBuF2u6d8jMPhgMPhuGRc13UGENEN7OsOkaT9LJjdbkdVVRU6OjrMsUQigY6ODni93nS3Q0QKKfkI1tjYiHXr1mH58uVYuXIlfv3rX2N4eBgPP/ywinaISBElAfSzn/0Mp0+fxpYtWxAIBHD77bfj3XffveTANBHNbkq+B3StwuEwnE4nDMPgMSCiG9B0X6O8FoyIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKpBxAe/fuxY9//GOUl5dD0zT8+c9/TpoXEWzZsgXz5s1DXl4eampqcOzYsaSac+fOYe3atdB1HcXFxVi/fj2GhoauaUOIKPOkHEDDw8NYtmwZtm/fPuX8tm3b8NJLL+Hll1/G/v37UVBQgNraWoyNjZk1a9euRW9vL9rb29HW1oa9e/fi0UcfvfqtIKLMJNcAgOzatctcTyQS4na75YUXXjDHBgcHxeFwyB//+EcREfnkk08EgBw8eNCseeedd0TTNDlx4sS0ntcwDAEghmFcS/tEdJ1M9zU6o8eAjh8/jkAggJqaGnPM6XSiuroafr8fAOD3+1FcXIzly5ebNTU1NbBYLNi/f/+UfzcajSIcDictRJT5ZjSAAoEAAMDlciWNu1wucy4QCKCsrCxpPicnByUlJWbNVzU3N8PpdJpLRUXFTLZNRIpkxFmwpqYmGIZhLgMDA6pbIqIZMKMB5Ha7AQDBYDBpPBgMmnNutxuhUChpfmJiAufOnTNrvsrhcEDX9aSFiDLfjAbQwoUL4Xa70dHRYY6Fw2Hs378fXq8XAOD1ejE4OIiuri6zZs+ePUgkEqiurp7JdojoBpeT6gOGhobw6aefmuvHjx9Hd3c3SkpK4PF4sGnTJjz33HO49dZbsXDhQjz99NMoLy/HmjVrAAC33XYb7r33XjzyyCN4+eWXMT4+joaGBvz85z9HeXn5jG0YEWWAVE+vvf/++wLgkmXdunUicv5U/NNPPy0ul0scDofcc8890tfXl/Q3zp49Kw888IAUFhaKruvy8MMPSyQSmfFTfESkxnRfo5qIiML8uyrhcBhOpxOGYfB4ENENaLqv0Yw4C0ZEsxMDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlIm5Z/lyVQiAolPQCQBS44dmqapboko6836AIpGziJy6u8YOTOA0S9Owmpz4Bv3/As0q011a0RZb9YH0Jm+D3Gyqw3nf74MsNrzEBv6ArnOMrWNEdHsPwZU6PomNMuFzUxMxBCPjSrsiIgmzfoAshcUA7hwvEcSCQyf/kxVO0R0kVkfQLZ8HQ699KIRQdQ4jQz8QViiWWfWB5DVUQBbfnHS2NDp45D4hJqGiMg06wMIAArLvpG0Hhs6B0kwgIhUSymAmpubsWLFChQVFaGsrAxr1qxBX19fUs3Y2Bh8Ph9KS0tRWFiI+vp6BIPBpJr+/n7U1dUhPz8fZWVl2Lx5MyYmrl8g5M6Zl7Qej40hNjx43Z6PiKYnpQDq7OyEz+fDvn370N7ejvHxcaxevRrDw8NmzRNPPIG3334bra2t6OzsxMmTJ3H//feb8/F4HHV1dYjFYvjwww/x2muvYceOHdiyZcvMbdVFNE1D3px5sNhyL/QQHcaYEbzCo4goLeQahEIhASCdnZ0iIjI4OCg2m01aW1vNmiNHjggA8fv9IiKye/dusVgsEggEzJqWlhbRdV2i0ei0ntcwDAEghmFMqz42EpaPX3tSDrz8iLkEevZIIpGY7qYSUQqm+xq9pmNAhmEAAEpKSgAAXV1dGB8fR01NjVmzaNEieDwe+P1+AIDf78eSJUvgcrnMmtraWoTDYfT29k75PNFoFOFwOGlJhSXHDntRadJY5NSxlP4GEc28qw6gRCKBTZs24c4778TixYsBAIFAAHa7HcXFxUm1LpcLgUDArLk4fCbnJ+em0tzcDKfTaS4VFRUp9WrJsSO/NPkxE6NhngkjUuyqA8jn8+Hw4cPYuXPnTPYzpaamJhiGYS4DAwMpPV7TtEsORI+c/QfGR1N7J0VEM+uqrgVraGhAW1sb9u7di/nz55vjbrcbsVgMg4ODSe+CgsEg3G63WXPgwIGkvzd5lmyy5qscDgccDsfVtGoqvGkBoGnAl19AlMQEL8kgUiyld0AigoaGBuzatQt79uzBwoULk+arqqpgs9nQ0dFhjvX19aG/vx9erxcA4PV60dPTg1AoZNa0t7dD13VUVlZey7ZckS3fiZzcQnM9ER/HcOj4dXs+Ivp6Kb0D8vl8eP311/HWW2+hqKjIPGbjdDqRl5cHp9OJ9evXo7GxESUlJdB1HRs3boTX68WqVasAAKtXr0ZlZSUefPBBbNu2DYFAAE899RR8Pt81v8u5EluBE/aCYkyMRs4PiCA2/AVEhPcGIlIkpXdALS0tMAwD3//+9zFv3jxzeeONN8yaF198ET/60Y9QX1+Pu+66C263G3/605/MeavVira2NlitVni9XvziF7/AQw89hGeffXbmtmoKmsUKW54zaSxy8u+QRPy6Pi8RXZ4mknlXZYbDYTidThiGAV3Xp/UYEUHwf9oxsO9Ncyy32I3K+qdgzbFfr1aJstJ0X6NZcS0YcP5MmK1gDqBd2OTx0QhiQ+cUdkWU3bImgACgoGwBrLYLx5ni0WGMD3+hsCOi7JZVAZRjz0NOblHS2JhxWlE3RJRVAWS15yGv5OakseHg//HmZESKZFUAQbPAdtF3gQCcPxXPSzKIlMiqANI0DXrFPyeNjZwdQDw2oqgjouyWVQEEnL9HtGaxmuuSmMBElAFEpELWBVBesfv86fgvxWNR/koGkSJZF0AWW+5XjgMJxocNHogmUiDrAkizWFHoviVpzPjHJ5j85VQiSp+sCyAAsOUlfzU8HhuBxHlNGFG6ZV0AaZqGQvc3oVkv3AggFjmLGL8RTZR2WRdAAGAvmAPNciGAJmKjmBgbUtgRUXbKygDKyStC3sW3aBXB6Bcn1TVElKWyMoAsOXbYCoovGhGMnOnnmTCiNMvKAAKAoq+cCRszQrwkgyjNsjaAHEVzk9ZHz51AYiKmqBui7JSVAaRpGhzOMljt+eZYIj6B8bGIwq6Isk9WBhAAOApLYLXnmevx6AhGz57gcSCiNMraANKsNuSXzr9oRBAb5u1ZidIpewPIYkVucfJPRIf/cURRN0TZKXsDSNOQP/efkm5SH4+N8md6iNIoawMIOH9rjot/lHDMCGJ8ZFBdQ0RZJqsDKCe3IOnnmuOxMd6cjCiNUvpp5kwkIhgeHsbExKVfMpQEYM2fg/ER48vaOL448SnGcy7/Q2qapqGoqAgWS1ZnN9GMmPUBBADr16/H3/72tynn/vXe21C7fOH5FRFs/8//wH/v6bvs3yoqKsJf/vIX3HzzzZetIaLpyYoAOnPmDE6cODHl3Ed9RbjrjiU4NrIcsUQubPnvIBj4ABPxxJT1uq4jznsHEc2IlD5HtLS0YOnSpdB1Hbquw+v14p133jHnx8bG4PP5UFpaisLCQtTX1yMYDCb9jf7+ftTV1SE/Px9lZWXYvHnzlB+P0uWz0AgOnrsLJ6O34sy4B1L2AG5yfUtZP0TZJKUAmj9/Pp5//nl0dXXh0KFDuPvuu/GTn/wEvb29AIAnnngCb7/9NlpbW9HZ2YmTJ0/i/vvvNx8fj8dRV1eHWCyGDz/8EK+99hp27NiBLVu2zOxWpeD4qQhOD+cBOH82LMdehMLCUmX9EGUVuUZz5syRP/zhDzI4OCg2m01aW1vNuSNHjggA8fv9IiKye/dusVgsEggEzJqWlhbRdV2i0ei0n9MwDAEghmF8bW0ikZC7775bcP6mz5cs+bm5svXf/kuefW6fbP33/fLCv78lty/61mXrdV2Xzz//PIU9RJR9pvsavepjQPF4HK2trRgeHobX60VXVxfGx8dRU1Nj1ixatAgejwd+vx+rVq2C3+/HkiVL4HJd+AZybW0tNmzYgN7eXnznO99JqYejR4+isLDwijXy5VmwyxmNjmHg6E7Y9F4cOTGC0ycO4H/7P79sfSKRwLFjxzA0xDsoEl3OdF8fKQdQT08PvF4vxsbGUFhYiF27dqGyshLd3d2w2+0oLi5Oqne5XAgEAgCAQCCQFD6T85NzlxONRhGNRs31cDgMADAMY1rHj65UIwK8svsgIAeRmMaFqCKCSCSCvLy8r60lylZX+p/+xVIOoG9/+9vo7u6GYRh48803sW7dOnR2dqbcYCqam5uxdevWS8arq6uh65f/zg5wPjCcTucVaxKJ6V8Bb7Vacccdd8Dj8Uz7MUTZZvJNwtdJ+dt0drsdt9xyC6qqqtDc3Ixly5bhN7/5DdxuN2KxGAYHB5Pqg8Eg3G43AMDtdl9yVmxyfbJmKk1NTTAMw1wGBgZSbZuIbkDX/HXeRCKBaDSKqqoq2Gw2dHR0mHN9fX3o7++H1+sFAHi9XvT09CAUCpk17e3t0HUdlZWVl30Oh8NhnvqfXIgo86X0EaypqQn33XcfPB4PIpEIXn/9dXzwwQd477334HQ6sX79ejQ2NqKkpAS6rmPjxo3wer1YtWoVAGD16tWorKzEgw8+iG3btiEQCOCpp56Cz+eDw+G4LhtIRDeulAIoFArhoYcewqlTp+B0OrF06VK89957+MEPfgAAePHFF2GxWFBfX49oNIra2lr8/ve/Nx9vtVrR1taGDRs2wOv1oqCgAOvWrcOzzz47s1v1Ffn5+TP2rknXdV4HRjRDNJHMuwdpOByG0+mEYRjTOgh9+vTppLNo10LTNLjdbuTkZMVVLERXZbqv0Vn/KtI0DWVlZarbIKIp8LMEESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUiZHdQNXQ0QAAOFwWHEnRDSVydfm5Gv1cjIygM6ePQsAqKioUNwJEV1JJBKB0+m87HxGBlBJSQkAoL+//4obR8nC4TAqKiowMDAAXddVt5MRuM+ujoggEomgvLz8inUZGUAWy/lDV06nk/9RXAVd17nfUsR9lrrpvDngQWgiUoYBRETKZGQAORwOPPPMM3A4HKpbySjcb6njPru+NPm682RERNdJRr4DIqLZgQFERMowgIhIGQYQESmTkQG0fft2LFiwALm5uaiursaBAwdUt6RMc3MzVqxYgaKiIpSVlWHNmjXo6+tLqhkbG4PP50NpaSkKCwtRX1+PYDCYVNPf34+6ujrk5+ejrKwMmzdvxsTERDo3RZnnn38emqZh06ZN5hj3WZpIhtm5c6fY7XZ55ZVXpLe3Vx555BEpLi6WYDCoujUlamtr5dVXX5XDhw9Ld3e3/PCHPxSPxyNDQ0NmzWOPPSYVFRXS0dEhhw4dklWrVsl3v/tdc35iYkIWL14sNTU18vHHH8vu3btl7ty50tTUpGKT0urAgQOyYMECWbp0qTz++OPmOPdZemRcAK1cuVJ8Pp+5Ho/Hpby8XJqbmxV2deMIhUICQDo7O0VEZHBwUGw2m7S2tpo1R44cEQDi9/tFRGT37t1isVgkEAiYNS0tLaLrukSj0fRuQBpFIhG59dZbpb29Xb73ve+ZAcR9lj4Z9REsFouhq6sLNTU15pjFYkFNTQ38fr/Czm4chmEAuHDBbldXF8bHx5P22aJFi+DxeMx95vf7sWTJErhcLrOmtrYW4XAYvb29aew+vXw+H+rq6pL2DcB9lk4ZdTHqmTNnEI/Hk/6lA4DL5cLRo0cVdXXjSCQS2LRpE+68804sXrwYABAIBGC321FcXJxU63K5EAgEzJqp9unk3Gy0c+dOfPTRRzh48OAlc9xn6ZNRAURX5vP5cPjwYfz1r39V3coNbWBgAI8//jja29uRm5urup2sllEfwebOnQur1XrJ2YhgMAi3262oqxtDQ0MD2tra8P7772P+/PnmuNvtRiwWw+DgYFL9xfvM7XZPuU8n52abrq4uhEIh3HHHHcjJyUFOTg46Ozvx0ksvIScnBy6Xi/ssTTIqgOx2O6qqqtDR0WGOJRIJdHR0wOv1KuxMHRFBQ0MDdu3ahT179mDhwoVJ81VVVbDZbEn7rK+vD/39/eY+83q96OnpQSgUMmva29uh6zoqKyvTsyFpdM8996Cnpwfd3d3msnz5cqxdu9b8Z+6zNFF9FDxVO3fuFIfDITt27JBPPvlEHn30USkuLk46G5FNNmzYIE6nUz744AM5deqUuYyMjJg1jz32mHg8HtmzZ48cOnRIvF6veL1ec37ylPLq1aulu7tb3n33Xbnpppuy6pTyxWfBRLjP0iXjAkhE5Le//a14PB6x2+2ycuVK2bdvn+qWlAEw5fLqq6+aNaOjo/LLX/5S5syZI/n5+fLTn/5UTp06lfR3PvvsM7nvvvskLy9P5s6dK08++aSMj4+neWvU+WoAcZ+lB2/HQUTKZNQxICKaXRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMv8PR7dOYw8kNOAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
